package com.changxianggu.crawler.cloud.test;

import cn.hutool.core.bean.BeanUtil;
import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import com.changxianggu.cloud.crawler.core.WebApplication;
import com.changxianggu.cloud.crawler.core.modules.storage.entity.CxgBook;
import com.changxianggu.cloud.crawler.core.modules.storage.service.ICxgBookService;
import com.changxianggu.cloud.crawler.core.utils.HttpUtil;
import com.changxianggu.cloud.crawler.core.utils.UserAgentUtil;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = {WebApplication.class})// 指定启动类
@Slf4j
public class RenYouJiaoYu {

    @Autowired
    ICxgBookService bookService;

    public static final String COOKIE = "AnonymousUserId=18a44492-28de-48b7-89a8-e078ca79cb96; neplayer_device_id=15813983797286462268894; BookMode=List; Hm_lvt_fc8003a996275a54c6ecc8d9e8637c5a=1581398380,1581415896,1582101988,1582267905; ASP.NET_SessionId=5q23hlnb5c0hntk4p32elqgc; __RequestVerificationToken=iz03krbbBmhIGICI_wF3vdFLYvPdlEc12VrqahPeWcEupjXz85OCMP5fRFE8gJgcr-uLt7AMHjj9Qq6l9-exRWsiaGFSGR0PPjrKXcJr_yE1; Hm_lpvt_fc8003a996275a54c6ecc8d9e8637c5a=1582268378";
    public static final String URL_PRE = "http://www.ryjiaoyu.com";


    @Test
    public void initCrawler() throws Exception {
        doCrawler();
    }

    public void doCrawler() throws Exception {
        int pageNum = 0;
        int totalPage = 321;
        for (int i = pageNum; i < totalPage; i++) {
            String url = "http://www.ryjiaoyu.com/book";
            String formatUrl = StrUtil.format(url, i);
            Map<String, Object> parameter = MapUtil.newHashMap(6);
            parameter.put("tab", "book");
            parameter.put("sort", "new");
            parameter.put("mode", "List");
            parameter.put("page", i);
            Map<String, Object> headers = MapUtil.newHashMap(5);
            headers.put("Cookie", COOKIE);
            headers.put("User-Agent", UserAgentUtil.random());
            String result = HttpUtil.doGet(formatUrl, headers, parameter);
            int tryCount = 1;
            while (result.contains("页面没有找到") && tryCount <= 5) {
                //睡眠2秒继续访问
                Thread.sleep(2000);
                result = HttpUtil.doPost(url, headers, parameter);
                tryCount++;
            }
            if (StrUtil.isNotBlank(result)) {
                Document htmlDoc = Jsoup.parse(result);
                Elements bookContentList = htmlDoc.select("#tab-book .block-books li");
                if (bookContentList != null && bookContentList.size() > 0) {
                    List<CxgBook> listBook = new ArrayList<>();
                    for (Element element : bookContentList) {
                        CxgBook bookEntity = new CxgBook();
                        String pkId = "";
                        Element aElement = element.select(".book-img a").first();
                        if (aElement != null) {
                            String href = aElement.attr("href");
                            pkId = href.substring(href.lastIndexOf("/") + 1);
                            bookEntity.setPkId(pkId);
                        }
                        CxgBook bookEntity1 = getBookEntity(pkId);
                        if (bookEntity1 == null) {
                            continue;
                        }
                        BeanUtil.copyProperties(bookEntity1, bookEntity);
                        bookEntity.setPkId(pkId);
                        Element imgEle = aElement.select("img").first();
                        if (imgEle != null) {
                            String src = imgEle.attr("src");
                            bookEntity.setCover(src);
                        }
                        Element titleEle = element.select(".book-info .name a").first();
                        if (titleEle != null) {
                            bookEntity.setBookName(titleEle.attr("title"));
                        }
                        Element authorEle = element.select(".book-info .author a").first();
                        if (authorEle != null) {
                            bookEntity.setAuthor(authorEle.text());
                        }
                        Element priceEle = element.select(".book-info .price").first();
                        if (priceEle != null) {
                            String substring = priceEle.text().substring(1);
                            bookEntity.setPrice(BigDecimal.valueOf(Double.parseDouble(substring)));
                        }
                        bookEntity.setIsSync(0);
                        bookEntity.setCreateTime(LocalDateTime.now());
                        bookEntity.setPressName("人民邮电出版社");
                        bookEntity.setStatus(1);
                        bookEntity.setPressId(37);
                        log.info(bookEntity.getBookName());
                        listBook.add(bookEntity);
                    }
                    bookService.saveBatch(listBook);
                    log.info("第" + i + "页数据保存成功");
                    log.warn("人邮社--->" + i);
                }
            }
        }
    }

    public CxgBook getBookEntity(String numId) throws Exception {
        CxgBook bookEntity = null;
        if (StrUtil.isBlank(numId)) {
            return null;
        }
        String url = "http://www.ryjiaoyu.com/book/details/" + numId;
        Map<String, Object> headers = MapUtil.newHashMap(5);
        headers.put("Cookie", COOKIE);
        headers.put("User-Agent", UserAgentUtil.random());
        String result = HttpUtil.doGet(url, headers);
        int tryCount = 1;
        while (result.contains("页面没有找到") && tryCount <= 5) {
            //睡眠2秒继续访问
            Thread.sleep(2000);
            result = HttpUtil.doGet(url, headers);
            tryCount++;
        }
        if (StrUtil.isNotBlank(result)) {
            Document htmlDoc = null;
            try {
                htmlDoc = Jsoup.parse(result);
            } catch (Exception e) {
                log.warn("jsoup解析html发生错误");
            }
            if (htmlDoc == null) {
                return null;
            }
            bookEntity = new CxgBook();
            Elements serieEles = htmlDoc.select(".book-series");
            if (CollUtil.isNotEmpty(serieEles)) {
                String awards = serieEles.get(0).text();
                bookEntity.setAwards(awards);
            }
            Elements introEles = htmlDoc.select(".intro");
            if (CollUtil.isNotEmpty(introEles)) {
                String intros = introEles.get(0).text();
                bookEntity.setBrief(intros);
            }
            Elements catalogEles = htmlDoc.select(".catalogue-main .bookmenu p");
            if (CollUtil.isNotEmpty(catalogEles)) {
                String catalog = catalogEles.get(0).text();
                bookEntity.setCatalog(catalog);
            }
            Elements linkEles = htmlDoc.select(".book-approaches .btn-sample");
            if (CollUtil.isNotEmpty(linkEles)) {
                String link = linkEles.get(0).attr("href");
                bookEntity.setDigitalUri("http://www.ryjiaoyu.com" + link);
            }
            Elements infoEles = htmlDoc.select(".publish-info li");
            if (CollUtil.isNotEmpty(infoEles)) {
                for (int i = 0; i < infoEles.size(); i++) {
                    Element element = infoEles.get(i);
                    String text = element.text();
                    if (StrUtil.isNotBlank(text)) {
                        if (text.startsWith("系列书名：")) {
                            bookEntity.setAwards(text.substring(5));
                        } else if (text.startsWith("出版日期：")) {
                            String substring1 = text.substring(5);
                            if (StrUtil.isNotBlank(substring1)) {
                                String pubDateStr = substring1 + " 00:00:00";
                                DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
                                LocalDateTime localDateTime = LocalDateTime.parse(pubDateStr, dtf);
                                bookEntity.setPubDate(localDateTime);
                            }
                        } else if (text.startsWith("书　　号：")) {
                            String isbn = text.substring(5);
                            if (StrUtil.isBlank(isbn)) {
                                return null;
                            }
                            bookEntity.setBookIsbn(isbn.replaceAll("-", "").replaceAll("—", ""));
                        } else if (text.startsWith("页　　数：")) {
                            String pages = text.substring(5);
                            String numbers = getNumbers(pages);
                            if (StrUtil.isNotBlank(numbers)) {
                                bookEntity.setPageNum(Integer.parseInt(numbers));
                            }
                        }
                    }
                }
            }
        }
        return bookEntity;
    }

    /**
     * 判断一个字符串是否都为数字
     *
     * @param strNum
     * @return
     */
    public boolean isDigit(String strNum) {
        Pattern pattern = Pattern.compile("[0-9]{1,}");
        Matcher matcher = pattern.matcher((CharSequence) strNum);
        return matcher.matches();
    }

    /**
     * 截取数字
     *
     * @param content
     * @return
     */
    public String getNumbers(String content) {
        Pattern pattern = Pattern.compile("\\d+");
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            return matcher.group(0);
        }
        return "";
    }
}
